#!/usr/bin/python
# __CR__
# Copyright (c) 2008-2014 EMC Corporation
# All Rights Reserved
#
# This software contains the intellectual property of EMC Corporation
# or is licensed to EMC Corporation from third parties.  Use of this
# software and the intellectual property contained therein is expressly
# limited to the terms and conditions of the License Agreement under which
# it is provided by or on behalf of EMC.
# __CR__

"""
Author: Atmos L4 team
Purpose: Run on the local node to check potential mds db corruption due to running mdssync. 
         In phase 1, mdssync.log is processed to check whether there is any unordered restoration
         of a big db file. This is done for local ports only. 
         In phase 2, first check the difference between the apparent size and the actual disk usage 
         of a db file, then for each db file split, the first few pages are read to check whethter          
         it is an empty split.
Usage: python %prog --help
Version: 1.0.0
Prerequisite :
Input/Output restriction:
"""

import os
import commands
import re
import sys

#SPLIT_FILE_SIZE_THRESHOLD = 20 * 1024 * 1024 * 1024
SPLIT_SIZE = 5 * 1024 * 1024 * 1024
PAGE_SIZE = 4 * 1024
PAGE_SIZE_METADATA = 16 * 1024
PAGE_COUNT = 10
MAX_SIZE_DIFF = 50 * 1024 * 1024

MDS_CONFIG_DIR = '/etc/maui/mds'
MDSSYNC_LOG = '/var/log/maui/mdssync.log'
WORK_DIR = os.path.basename(__file__) + '.work'

DEBUG = False 

def print_err(*args):
    """
    print error message to stderr
    """
    sys.stderr.write(' '.join(map(str, args)) + '\n')

def find_last_run(log_file, port_pid_dict):
    """
    filter mdssync log to find the last successful run for each local port
    """
    succeeded = set()
    log_file_pid = os.path.join(work_dir, os.path.basename(log_file) + '.pid')
    
    cmd = 'grep -i -e "sync MDS for port" -e "\[TmpEnv\] Clean up" ' + log_file + ' | tac > ' + log_file_pid
    status, _ = commands.getstatusoutput(cmd)
    if status != 0:
        print_err("Error: failed to process " + log_file)
        return status
        
    try:
        f = open(log_file_pid, 'r')
        for line in f.readlines():
            words = line.split(' ')
            pid = words[4]
            # the line indicates that mdssync completed successfully
            if (line.lower()).find("[TmpEnv] Clean up".lower()) >= 0:
                succeeded.add(pid)
            # the line contains port info
            else:
                port = words[-2]
                if not port.isdigit():
                    print "Warning: " + port + " is not a digit, skipping it"
                    continue
                # check block restoration order for local port only
                elif port.find("104") == 0:
                    if (port not in port_pid_dict.keys()) and (pid in succeeded):
                        port_pid_dict[port] = pid
        f.close()
    except IOError:
        print_err("Error: cannot open/read " + log_file_pid)
    except (ValueError, IndexError):
        print_err("Error: unsupported mdssync log format found in " + log_file_pid)
    
    return 0
        
    
    
def process_log_file(log_file):
    """
    for local ports, check the order of block restoration of each SplitMerge file in mdssync log
    """
    # map from mds ports to the last mdssync pid
    port_pid_dict = {}
    # inverse map of port_pid_dict
    pid_port_dict = {}
    split_dict = {}
    failed_split_set = set()
    
    if DEBUG:
        print "DEBUG: processing " + log_file
    
    # return if find_last_run failed or no successful mdssync is detected
    if (find_last_run(log_file, port_pid_dict) != 0) or (not port_pid_dict):
        return
    pid_port_dict = dict((v, k) for k, v in port_pid_dict.items())
    
    log_file_restore = os.path.join(work_dir, os.path.basename(log_file) + '.restore')
    cmd = 'grep -i -e "\[SplitMerge\] Restore block" ' + log_file + ' > ' + log_file_restore
    status, _ = commands.getstatusoutput(cmd)
    if status != 0:
        # grep exits with 1 if no pattern found, the high byte of status is the exit status 
        if (status >> 8) == 1:
            print "No use of SplitMerge found in " + log_file + ", skip it"
        else: 
            print_err("Error: failed to process " + log_file)
        return
    
    try:
        f = open(log_file_restore, 'r')
        for line in f.readlines():
            words = line.split(' ')
            pid = words[4]
            if pid not in pid_port_dict.keys():
                continue
            
            # block is like split_name/block-xx.gz
            block = words[-4]
            block_name = block[0:block.find('/')]
            block_id = int(block[block.rfind('-')+1 : block.rfind('.')])
            
            split_key = pid_port_dict[pid] + ' ' + pid + ' ' + block_name
            # skip the block if the split is already in failed list
            if split_key in failed_split_set:
                continue
            # initialize the split_dict counter if it is a new split
            elif split_key not in split_dict:
                split_dict[split_key] = 0
            
            # wrong order if block_id mismatches the counter in split_dict
            if block_id != split_dict[split_key]:
                failed_split_set.add(split_key)
            else:
                split_dict[split_key] += 1
        f.close()
    except IOError: 
        print_err("Error: cannot open/read " + log_file_restore)
    except (ValueError, IndexError):
        print_err("Error: unsupported mdssync log format found in " + log_file_restore)

    if failed_split_set:
        print "=== In file " + log_file + " ==="
        for split_key in failed_split_set:
            print split_key + ", block idx: ", split_dict[split_key]
            failed_port = split_key.split(' ')[0]
            mds_ports[failed_port] = False

def check_zero(data):
    """
    check whether data are all zero
    """
    for char in data:
        if char != '\x00':
            return False
    return True

def process_db_file(db_file, port):
    """
    check 1. the diff between apparent size and actual disk usage
    of a db_file; 2. whether there is a hole in it.
    """
    if DEBUG:
        print "DEBUG: processing " + db_file
        
    offset = 0
    f = None
    try:
        file_size = os.path.getsize(db_file)
        cmd = "du --block-size=1 " + db_file + " | cut -f 1" 
        status, disk_size = commands.getstatusoutput(cmd)
        if status != 0 or (not disk_size.isdigit()):
            print_err("Error: failed to get disk usage of " + db_file)
        else:
            if (file_size - int(disk_size)) > MAX_SIZE_DIFF:
                print port + " " + db_file + ": disk usage(" + disk_size + ") is well below apparent size(" + str(file_size) + ")"
                mds_ports[port] = False
                return 
                
        f = open(db_file, 'rb')
        if db_file.find("metadata"):
            # page size of metadata db is 16k, others are 4k
            page_size = PAGE_SIZE_METADATA
        else:
            page_size = PAGE_SIZE
            
        while offset < file_size:
            f.seek(offset) 
            for page_idx in range(PAGE_COUNT):
                content = f.read(page_size)
                # f.read() would return empty string if EOF is reached
                if not content:
                    break
                if check_zero(content):
                    print port + " " + db_file + ': found zero page at split idx: ' + str(offset/SPLIT_SIZE) + ', page idx: ' + str(page_idx)
                    mds_ports[port] = False
                    break
            offset += SPLIT_SIZE
            
    except (OSError, IOError):
        print_err("Error: cannot open/read " + db_file)
        
    if f:
        f.close()


def filter_files_in_db_env(directory, pattern):
    """
    filter files in a dir according to given pattern
    """
    try:
        files = os.listdir(directory)
    except OSError:
        print_err("Error: cannot read directory: " + directory)
        return []
    
    output = []
    for db_file in files:
        if re.match(pattern, db_file):
            db_file_full_path = os.path.join(directory, db_file)
            """
            # comment out, SPLIT_FILE_SIZE_THRESHOLD may be different across mdssync versions
            try:
                file_size = os.path.getsize(db_file_full_path)
            except OSError:
                print_err("Error: failed to get the size of " + db_file_full_path)
                continue
                
            if file_size >= SPLIT_FILE_SIZE_THRESHOLD:
                output.append(db_file_full_path)
            """
            output.append(db_file_full_path)
    return output

def print_ports_status():
    """
    print the current status of each mds port to be checked
    """
    print "MDS status:"
    for port, val in mds_ports.items():
        if val:
            status = "Passed"
        else:
            status = "Failed"
        print port + " : " + status
    
if __name__ == '__main__':

    mds_ports = {}
    ports = os.listdir(MDS_CONFIG_DIR)
    for port in ports:
        if port.isdigit():
            mds_ports[port] = True
        else:
            print "Warning: skipping non numeric folders in " + MDS_CONFIG_DIR
    
    print "Ports to be checked:",
    for port in  mds_ports.keys():
        print port,
    print
    
    work_dir = os.path.realpath(WORK_DIR)
    if not os.path.exists(work_dir):
        os.mkdir(work_dir)
    
    print "===== Phase 1: Check mdssync.log for incorrect restore order ====="
    process_log_file(MDSSYNC_LOG)
    
    print_ports_status()
    print "===== Phase 1: End ====="
    
    print "===== Phase 2: Check db env for zero pages in db files ====="
    for port, val in mds_ports.items():
        if not val:
            continue
        
        cmd = 'grep EnvRoot /etc/maui/mds/' + port + '/mds_cfg.xml | awk -F\\\" \'{ print $4 }\''
        status, db_env_dir = commands.getstatusoutput(cmd)
        
        if status != 0 or db_env_dir.find("No such file or directory") >= 0:
            print_err("Error: failed to get db env for port " + port + ", is it not configured?")
            # if error, assume that the port is not configured, skip it.
            mds_ports[port] = True
        elif not os.path.exists(db_env_dir):
            print_err("Error: db env missing for port " + port)
            mds_ports[port] = False
        else:
            db_files = filter_files_in_db_env(db_env_dir, "(.*bdbdb$)|(.*bdbxml$)")
            for db_file_file in db_files:
                process_db_file(db_file_file, port)

    print_ports_status()
    print "===== Phase 2: End ====="
    
    print "===== Summary ====="
    passed_ports = []
    failed_ports = []
    for port, val in mds_ports.items():
        if val:
            passed_ports.append(port)
        else:
            failed_ports.append(port)
    
    print "Need to run mdssync for ports: ",
    for port in failed_ports:
        print port,
    print
    print "Need to run db_verify for ports: ",
    for port in passed_ports:
        print port,
    print
    print "Please manually clean the temp work dir: " + work_dir
    print "===== Summary End ====="
    
